param([string]$rootDirectory) $archiveDataTypes = ( "Event", "Event Native", "Event Comtrade", "Flicker Trend", "PQ Trend", "RMS Trend", "SER" ) $endStamp = "7355cabf-be21-4b07-a8c3-a70fb54a7d6c" function Load-FolderList { $directory = Get-Location $path = [System.IO.Path]::Combine($directory, "archive-cleanup.folders.lst") if ([System.IO.File]::Exists($path)) { $folderList = [System.IO.File]::ReadAllLines($path) $lastLine = $folderList | Select-Object -Last 1 if ($lastLine -eq $endStamp) { $folderCount = $folderList.Length - 1 $folderList | Select-Object -First $folderCount return } } $folderList = ,[System.IO.Path]::Combine($rootDirectory, "DFR") | ForEach-Object { [System.IO.Directory]::GetDirectories($_, "*", "TopDirectoryOnly") } | # device type ForEach-Object { [System.IO.Directory]::GetDirectories($_, "*", "TopDirectoryOnly") } | # station ForEach-Object { [System.IO.Directory]::GetDirectories($_, "*", "TopDirectoryOnly") } | # data type Where-Object { $archiveDataTypes.Contains([System.IO.Path]::GetFileName($_)) } $folderList > $path $endStamp | Out-File $path -Append -NoNewline $folderList } function Load-Checkpoint { $directory = Get-Location $checkpoint = [System.IO.Directory]::GetFiles($directory, "*.chk", "TopDirectoryOnly") | ForEach-Object { [System.IO.Path]::GetFileNameWithoutExtension($_) } | Where-Object { [int]::TryParse($_, [ref]$null) } | ForEach-Object { [int]$_ } | Sort-Object -Descending | Select-Object -First 1 if ($checkpoint -eq $null) { $checkpoint = 0 } $checkpoint } function Load-FileList($checkpoint, $folderPath) { $directory = Get-Location $fileListPath = [System.IO.Path]::Combine($directory, "$checkpoint.archive-cleanup.files.lst") if ([System.IO.File]::Exists($fileListPath)) { $fileList = [System.IO.File]::ReadAllLines($fileListPath) $lastLine = $fileList | Select-Object -Last 1 if ($lastLine -eq $endStamp) { $fileCount = $fileList.Length - 1 Select-Object -First $fileCount | ForEach-Object { ConvertFrom-Json $_ } return } } $sourceDirectories = ,$folderPath $archiveDirectory = [System.IO.Path]::Combine($folderPath, "Archive") if ([System.IO.Directory]::Exists($archiveDirectory)) { $sourceDirectories = $sourceDirectories + ,$archiveDirectory + ( [System.IO.Directory]::GetDirectories($archiveDirectory, "*", "TopDirectoryOnly") | Where-Object { [System.IO.Path]::GetFileName($_) -match "^\d\d\d\d$" } ) } $destinationLookup = @{} $fileList = ForEach ($sourceDirectory in $sourceDirectories) { $sourcePatterns = @([System.IO.Directory]::GetFiles($sourceDirectory, "*", "TopDirectoryOnly") | ForEach-Object { ,[System.IO.Path]::ChangeExtension($_, "*") } | Sort-Object | Get-Unique) ForEach ($sourcePattern in $sourcePatterns) { $sourceName = [System.IO.Path]::GetFileNameWithoutExtension($sourcePattern) $dateField = $sourceName.Split(",")[0] $sourceDate = [System.DateTime]::MinValue if (-not [System.DateTime]::TryParseExact($dateField, "yyMMdd", $null, "None", [ref]$sourceDate)) { $sourceDate = [System.IO.Directory]::GetFiles($sourceDirectory, "$sourceName.*", "TopDirectoryOnly") | ForEach-Object { [System.IO.File]::GetLastWriteTime($_).Date } | Select-Object -First 1 } $dateDiff = [System.DateTime]::Today.Subtract($sourceDate).TotalDays if ($dateDiff -ge 0 -and $dateDiff -le 90) { continue } $archiveFolder = "{0:yyyy-MM}" -f $sourceDate if ($sourceDate -lt [System.DateTime]::new(2000, 01, 01)) { $archiveFolder = "Time Errors" } if ($sourceDate -gt [System.DateTime]::Today.AddDays(1)) { $archiveFolder = "Time Errors" } $archiveFolderPath = [System.IO.Path]::Combine($archiveDirectory, $archiveFolder) $destinationNameArgs = ($sourceName, 1) $destinationName = $sourceName $destinationKey = [System.IO.Path]::Combine($archiveFolder, $destinationName) while ($destinationLookup[$destinationKey]) { $destinationNameArgs[1]++ $destinationName = "{0} ({1})" -f $destinationNameArgs $destinationKey = [System.IO.Path]::Combine($archiveFolder, $destinationName) } $destinationLookup[$destinationKey] = $true [System.IO.Directory]::GetFiles($sourceDirectory, "$sourceName.*", "TopDirectoryOnly") | Where-Object { [System.IO.Path]::GetFileNameWithoutExtension($_) -eq $sourceName } | ForEach-Object { $source = $_ $extension = [System.IO.Path]::GetExtension($source) $destination = [System.IO.Path]::Combine($archiveFolderPath, "$destinationName$extension") [pscustomobject]@{ Source = $source ; Destination = $destination } } } } [System.IO.Directory]::GetFiles($directory, "*.files.lst", "TopDirectoryOnly") | ForEach-Object { [System.IO.File]::Delete($_) } $fileList | ForEach-Object { ConvertTo-Json $_ -Compress } > $fileListPath $endStamp | Out-File $fileListPath -Append -NoNewline $fileList } function Update-Checkpoint($checkpoint) { $directory = Get-Location $previous = [System.IO.Directory]::GetFiles($directory, "*.chk", "TopDirectoryOnly") | ForEach-Object { [System.IO.Path]::GetFileNameWithoutExtension($_) } | Where-Object { [int]::TryParse($_, [ref]$null) } | ForEach-Object { [int]$_ } | Sort-Object -Descending | Select-Object -First 1 if ($previous -eq $null) { $null > "$checkpoint.chk" return } $previousPath = [System.IO.Path]::Combine($directory, "$previous.chk") $checkpointPath = [System.IO.Path]::Combine($directory, "$checkpoint.chk") [System.IO.File]::Move($previousPath, $checkpointPath) } $folderList = @(Load-FolderList) $checkpoint = Load-Checkpoint $total = $folderList.Length while ($checkpoint -lt $folderList.Length) { $folder = $folderList[$checkpoint] $progress = $checkpoint / $total * 100 Write-Host ("[{0:N2}%] ({1}/{2}) Working on {3}..." -f $progress, $checkpoint, $total, $folder) $fileList = @(Load-FileList $checkpoint $folder) ForEach ($file in $fileList) { $source = $file.Source $destination = $file.Destination if (-not [System.IO.File]::Exists($source)) { continue } $destinationFolder = [System.IO.Path]::GetDirectoryName($destination) [System.IO.Directory]::CreateDirectory($destinationFolder) | Out-Null try { [System.IO.File]::Move($source, $destination) } catch { Write-Error ("", "$source", "$destination", "$_" -Join "`n") } } [System.IO.Directory]::GetDirectories($folder, "*", "AllDirectories") | ForEach-Object { $dir = $_ $enumerator = [System.IO.Directory]::EnumerateFileSystemEntries($dir, "*", "TopDirectoryOnly") $file = $enumerator | Select-Object -First 1 $enumerator.Dispose() if (-not $file) { try { [System.IO.Directory]::Delete($dir) } catch { Write-Error ("", "$dir", "$_" -Join "`n") } } } $checkpoint++ Update-Checkpoint $checkpoint }